﻿using System;
using System.Collections;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;

namespace SimpleCrawler
{
    class Crawler
    {
        private Hashtable urls = new Hashtable();
        private int count = 0;
        public Action<string> miao;
        public string HostFilter { get; set; } //主机过滤规则
        public string FileFilter { get; set; } //文件过滤规则
        /*static void Main(string[] args)
        {
            SimpleCrawler myCrawler = new SimpleCrawler();
            string startUrl = "http://www.cnblogs.com/dstang2000/";
            if (args.Length >= 1) startUrl = args[0];
            myCrawler.urls.Add(startUrl, false);//加入初始页面
            new Thread(myCrawler.Crawl).Start();
        }*/

        public void startCrawler(string startUrl)
        {
            urls.Add(startUrl, false);//加入初始页面
            miao("开始爬行了.... \n");
            for (int i = 0; i < 5; i++)
            {
                new Thread(this.Crawl).Start();
            }
        }
        private void Crawl()
        {
            //Console.WriteLine("开始爬行了.... ");
            //miao("开始爬行了.... \n");
            while (true)
            {
                string current = null;
                lock (urls)
                {
                    foreach (string url in urls.Keys)
                    {
                        if ((bool)urls[url]) continue;
                        current = url;
                    }
                }
                

                if (current == null || count > 10) break;
                //Console.WriteLine("爬行" + current + "页面!");
                miao("爬行" + current + "页面! \n");
                string html = DownLoad(current); // 下载
                urls[current] = true;
                count++;
                Parse(html, current);//解析,并加入新的链接
                //Console.WriteLine("爬行结束");
                miao("爬行结束\n");
            }
        }

        public string DownLoad(string url)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);
                string fileName = count.ToString();
                File.WriteAllText(fileName, html, Encoding.UTF8);
                return html;
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                miao(ex.Message + "\n");
                return "";
            }
        }

        private void Parse(string html, string startUrl)
        {
            string strRef = @"(href|HREF)[]*=[]*[""'][^""'#>]+[""']";
            //Console.WriteLine(1);
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1)
                          .Trim('"', '\"', '#', '>');
                //strRef = match.Groups["url"].Value;
                //Console.WriteLine(strRef);
                if (strRef == null || strRef == "") continue;
                strRef = FixUrl(strRef, startUrl);
                //if (strRef.Length == 0) continue;
                Match strRefMatch = Regex.Match(strRef, @"^(?<site>https?://(?<host>[\w.-]+)(:\d+)?($|/))(\w+/)*(?<file>[^#?]*)");
                string host = strRefMatch.Groups["host"].Value;
                string file = strRefMatch.Groups["file"].Value;
                Match startMatch = Regex.Match(startUrl, @"^(?<site>https?://(?<host>[\w.-]+)(:\d+)?($|/))(\w+/)*(?<file>[^#?]*)");
                HostFilter = startMatch.Groups["host"].Value;
                FileFilter = @"[^#?]*.(htm|html|aspx|php|jsp)[#?]*";
                //Console.WriteLine(host);
                //Console.WriteLine(file);
                //Console.WriteLine(HostFilter);
                //Console.WriteLine(FileFilter);
                if (urls[strRef] == null&&Regex.IsMatch(host, HostFilter) && (Regex.IsMatch(file, FileFilter)||file==""))
                {
                    //Console.WriteLine(2);
                    miao("找到" + strRef + "页面! \n");
                    urls[strRef] = false;
                }
                //if (urls[strRef] == null) urls[strRef] = false;
            }

        }

        private string FixUrl(string url, string pageUrl)
        {
            if (url.Contains("://"))
            {
                return url;
            }
            if (url.StartsWith("/"))
            {
                Match urlMatch = Regex.Match(pageUrl, @"^(?<site>https?://(?<host>[\w.-]+)(:\d+)?($|/))(\w+/)*(?<file>[^#?]*)");
                String site = urlMatch.Groups["site"].Value;
                return site.EndsWith("/") ? site + url.Substring(1) : site + url;
            }

            if (url.StartsWith("../"))
            {
                url = url.Substring(3);
                int index = pageUrl.LastIndexOf('/');
                return FixUrl(url, pageUrl.Substring(0, index));
            }

            if (url.StartsWith("./"))
            {
                return FixUrl(url.Substring(2), pageUrl);
            }
            if (url.Contains("://"))
            {
                return url;
            }
            if (url.StartsWith("//"))
            {
                return "http:" + url;
            }

            int end = pageUrl.LastIndexOf("/");
            return pageUrl.Substring(0, end) + "/" + url;
        }
    }
}
